package edu.ptit.jad.preprocess.app;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import java.io.StringReader;
import java.util.List;

import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;

import edu.ptit.jad.configuration.CONSTANTS;
import edu.ptit.jad.data.DataAccess;
import edu.ptit.jad.exception.AdException;
import edu.ptit.jad.preprocess.analysis.SynonymAnalyzer;
import edu.ptit.jad.preprocess.analysis.SynonymEngine;
import edu.ptit.jad.preprocess.analysis.VietnameseSynonymEngine;

public abstract class Filter {

	private String output;
	static List<String> stopWords;

	// private String input;

	public Filter(String output) {
		// setInput(input);
		setOutput(output);

		System.out.println("stopwords loading....");
		try {
			stopWords = DataAccess.getInstance().getStopWords();
		} catch (AdException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		System.out.println("news and ads processing....");

	}

	public void filter() throws Exception {

		VietnameseSynonymEngine engine = new VietnameseSynonymEngine();
		SynonymEngine sengine = new VietnameseSynonymEngine();
		SynonymAnalyzer analyzer = new SynonymAnalyzer(sengine);

		File file = new File(getOutput());
		BufferedWriter out = new BufferedWriter(new OutputStreamWriter(
				new FileOutputStream(file), "UTF8"));

		List<String> docs = getListDocs();
		int current = 0;

		out.write("000" + docs.size() + "\n");

		for (String doc : docs) {
			TokenStream stream = analyzer.tokenStream("contents",
					new StringReader(doc));
			TermAttribute term = stream.addAttribute(TermAttribute.class);
			TypeAttribute type = stream.addAttribute(TypeAttribute.class);
			PositionIncrementAttribute posIncr = stream
					.addAttribute(PositionIncrementAttribute.class);

			int position = 0;
			while (stream.incrementToken()) {
				int increment = posIncr.getPositionIncrement();
				if (increment > 0) {
					position = position + increment;
					String termStr = this.specialFilter(engine
							.getSynonymsTerm(term.term()));

					termStr = removeNumber(termStr);
					if (!termStr.trim().equals("")
							&& !stopWords.contains(termStr.trim()))
						out.write(termStr + " ");
					// if(termStr=="có")
					// System.err.println(stopWords.contains("có"));

				}

			}

			out.write("\n");
			current++;

			System.out.println(current + " done!");

		}

		System.out.println("Completed!");
		out.close();

	}

	private String specialFilter(String term) {

		return term
				.replaceAll(
						"\\”|\\“|\\/|\\=|\\`|\\.|\\>|\\<|\\?|\\'|\\[|\\]|\\'|\\+|\\-|\\(|\\)|\\{|\\}|\\,|\\'|\\;|\\%|\\~|\\@|\\$|\\&|\\*|\\!",
						"").replaceAll("\\d*$", "");

	}

	private String removeNumber(String str) {
		StringBuilder ans = new StringBuilder();
		char currentChar;
		for (int i = 0; i < str.length(); ++i) {
			currentChar = str.charAt(i);
			if (!Character.isDigit(currentChar)) {
				ans.append(currentChar);
			}
		}

		return ans.toString();

	}

	abstract List<String> getListDocs();

	public String getOutput() {
		return output;
	}

	private void setOutput(String output) {
		this.output = output;
	}

}
